{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入所需库\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import jieba\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "# 模拟训练数据\n",
    "def generate_sample_data():\n",
    "    return [\n",
    "        \"打开空调\\t启动空调\\t1\",\n",
    "        \"打开空调\\t关闭灯光\\t0\",\n",
    "        \"播放周杰伦的歌\\t我想听周杰伦的音乐\\t1\",\n",
    "        # 更多示例数据...\n",
    "    ]\n",
    "\n",
    "# 保存到临时文件\n",
    "with open('query_pairs.txt', 'w', encoding='utf-8') as f:\n",
    "    f.write('\\n'.join(generate_sample_data()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache C:\\Users\\honor\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 3.644 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    }
   ],
   "source": [
    "# 数据加载函数\n",
    "def load_data(filepath):\n",
    "    data = pd.read_csv(filepath, sep='\\t', header=None, names=['query1', 'query2', 'label'])\n",
    "    return data\n",
    "\n",
    "# 中文分词处理器\n",
    "def chinese_tokenizer(text):\n",
    "    english_words = re.findall(r'[a-zA-Z]+', text)\n",
    "    chinese_parts = jieba.cut(text)\n",
    "    return [part for part in chinese_parts if part.strip()] + english_words\n",
    "\n",
    "# 数据预处理流程\n",
    "def preprocess_data(df):\n",
    "    df['q1_tokens'] = df['query1'].apply(lambda x: ' '.join(chinese_tokenizer(x)))\n",
    "    df['q2_tokens'] = df['query2'].apply(lambda x: ' '.join(chinese_tokenizer(x)))\n",
    "    return df\n",
    "\n",
    "# 执行预处理\n",
    "df = load_data('query_pairs.txt')\n",
    "processed_df = preprocess_data(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vectorize_text(df):\n",
    "    tokenizer = Tokenizer()\n",
    "    tokenizer.fit_on_texts(pd.concat([df['q1_tokens'], df['q2_tokens']]))\n",
    "    \n",
    "    q1_seq = tokenizer.texts_to_sequences(df['q1_tokens'])\n",
    "    q2_seq = tokenizer.texts_to_sequences(df['q2_tokens'])\n",
    "    \n",
    "    # 正确计算 max_len：分别计算两个序列的最大长度再取最大值\n",
    "    max_len_q1 = max(len(seq) for seq in q1_seq) if q1_seq else 0\n",
    "    max_len_q2 = max(len(seq) for seq in q2_seq) if q2_seq else 0\n",
    "    max_len = max(max_len_q1, max_len_q2)\n",
    "    \n",
    "    q1_padded = pad_sequences(q1_seq, maxlen=max_len, padding='post')\n",
    "    q2_padded = pad_sequences(q2_seq, maxlen=max_len, padding='post')\n",
    "    \n",
    "    return q1_padded, q2_padded, np.array(df['label']), tokenizer, max_len\n",
    "\n",
    "X1, X2, y, tokenizer, max_len = vectorize_text(processed_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\honor\\AppData\\Roaming\\Python\\Python312\\site-packages\\keras\\src\\backend\\tensorflow\\core.py:232: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 共享的LSTM编码器\n",
    "def create_encoder(vocab_size, embedding_dim, max_len):\n",
    "    input_layer = tf.keras.layers.Input(shape=(max_len,))\n",
    "    x = tf.keras.layers.Embedding(vocab_size, embedding_dim)(input_layer)\n",
    "    x = tf.keras.layers.LSTM(128)(x)\n",
    "    return tf.keras.Model(inputs=input_layer, outputs=x)\n",
    "\n",
    "# 孪生网络架构\n",
    "def build_siamese_model(vocab_size, max_len, embedding_dim=128):\n",
    "    encoder = create_encoder(vocab_size, embedding_dim, max_len)\n",
    "    \n",
    "    input1 = tf.keras.layers.Input(shape=(max_len,))\n",
    "    input2 = tf.keras.layers.Input(shape=(max_len,))\n",
    "    \n",
    "    encoded1 = encoder(input1)\n",
    "    encoded2 = encoder(input2)\n",
    "    \n",
    "    distance = tf.keras.layers.Lambda(\n",
    "        lambda x: tf.keras.backend.abs(x[0] - x[1]))([encoded1, encoded2])\n",
    "    \n",
    "    output = tf.keras.layers.Dense(1, activation='sigmoid')(distance)\n",
    "    \n",
    "    model = tf.keras.Model(inputs=[input1, input2], outputs=output)\n",
    "    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "model = build_siamese_model(len(tokenizer.word_index)+1, max_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m14s\u001b[0m 14s/step - accuracy: 0.5000 - loss: 0.6947 - val_accuracy: 0.0000e+00 - val_loss: 0.6938\n",
      "Epoch 2/50\n",
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 534ms/step - accuracy: 1.0000 - loss: 0.6845 - val_accuracy: 0.0000e+00 - val_loss: 0.6941\n",
      "Epoch 3/50\n",
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 497ms/step - accuracy: 1.0000 - loss: 0.6753 - val_accuracy: 0.0000e+00 - val_loss: 0.6945\n",
      "Epoch 4/50\n",
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 497ms/step - accuracy: 1.0000 - loss: 0.6671 - val_accuracy: 0.0000e+00 - val_loss: 0.6951\n",
      "Epoch 5/50\n",
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 457ms/step - accuracy: 1.0000 - loss: 0.6597 - val_accuracy: 0.0000e+00 - val_loss: 0.6958\n",
      "Epoch 6/50\n",
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 397ms/step - accuracy: 1.0000 - loss: 0.6517 - val_accuracy: 0.0000e+00 - val_loss: 0.6966\n"
     ]
    }
   ],
   "source": [
    "# 数据拆分\n",
    "X1_train, X1_test, X2_train, X2_test, y_train, y_test = train_test_split(\n",
    "    X1, X2, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# 训练配置\n",
    "early_stopping = tf.keras.callbacks.EarlyStopping(\n",
    "    monitor='val_loss', patience=5, restore_best_weights=True)\n",
    "\n",
    "# 执行训练\n",
    "history = model.fit(\n",
    "    [X1_train, X2_train],\n",
    "    y_train,\n",
    "    validation_data=([X1_test, X2_test], y_test),\n",
    "    epochs=50,\n",
    "    batch_size=8,\n",
    "    callbacks=[early_stopping]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 1s/step\n",
      "预测结果: 1 (置信度: 0.5003)\n"
     ]
    }
   ],
   "source": [
    "# 评估指标\n",
    "def evaluate_model(model, X1_test, X2_test, y_test):\n",
    "    y_pred = (model.predict([X1_test, X2_test]) > 0.5).astype(int)\n",
    "    print(classification_report(y_test, y_pred))\n",
    "    plot_confusion_matrix(y_test, y_pred)\n",
    "\n",
    "# 预测接口\n",
    "def predict_match(query_pair, model, tokenizer, max_len):\n",
    "    q1, q2 = query_pair\n",
    "    seq1 = tokenizer.texts_to_sequences([' '.join(chinese_tokenizer(q1))])\n",
    "    seq2 = tokenizer.texts_to_sequences([' '.join(chinese_tokenizer(q2))])\n",
    "    \n",
    "    padded1 = pad_sequences(seq1, maxlen=max_len, padding='post')\n",
    "    padded2 = pad_sequences(seq2, maxlen=max_len, padding='post')\n",
    "    \n",
    "    proba = model.predict([padded1, padded2])[0][0]\n",
    "    return int(proba > 0.5), float(proba)\n",
    "\n",
    "# 示例预测\n",
    "test_case = (\"打开蓝牙\", \"开启蓝牙\")\n",
    "label, proba = predict_match(test_case, model, tokenizer, max_len)\n",
    "print(f\"预测结果: {label} (置信度: {proba:.4f})\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 模型保存\n",
    "def save_artifacts(model, tokenizer, max_len):\n",
    "    model.save('semantic_match_model.h5')\n",
    "    with open('tokenizer.pkl', 'wb') as f:\n",
    "        pickle.dump(tokenizer, f)\n",
    "    with open('max_len.pkl', 'wb') as f:\n",
    "        pickle.dump(max_len, f)\n",
    "\n",
    "# 模型加载\n",
    "def load_artifacts():\n",
    "    model = tf.keras.models.load_model('semantic_match_model.h5')\n",
    "    with open('tokenizer.pkl', 'rb') as f:\n",
    "        tokenizer = pickle.load(f)\n",
    "    with open('max_len.pkl', 'rb') as f:\n",
    "        max_len = pickle.load(f)\n",
    "    return model, tokenizer, max_len"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
